{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:44:11.185370Z",
     "start_time": "2025-07-17T00:44:06.215817Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from modelscope.msdatasets import MsDataset\n",
    "\n",
    "dataset = MsDataset.load('train.jsonl')\n",
    "data = dataset.to_hf_dataset().select(range(10))\n",
    "data[:10]"
   ],
   "id": "c2912e2519e3352d",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dadudu/miniconda3/envs/mini-gpt/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2025-07-17 08:44:07,475 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from train.jsonl. Please make sure that you can trust the external codes.\n",
      "2025-07-17 08:44:07,476 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from json. Please make sure that you can trust the external codes.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'question': ['你好，最近怎么样？',\n",
       "  '今天天气如何？',\n",
       "  '你喜欢旅行吗？',\n",
       "  '你最喜欢的食物是什么？',\n",
       "  '你有什么兴趣爱好？',\n",
       "  '你最喜欢的电影是什么？',\n",
       "  '你喜欢听音乐吗？',\n",
       "  '你最喜欢的季节是哪个？',\n",
       "  '你有什么宠物吗？',\n",
       "  '你有宠物吗？'],\n",
       " 'answer': ['你好！我最近还不错，谢谢。',\n",
       "  '今天的天气很晴朗。',\n",
       "  '是的，我非常喜欢旅行。',\n",
       "  '我最喜欢的食物是寿司。',\n",
       "  '我喜欢阅读和运动。',\n",
       "  '我最喜欢的电影是《肖申克的救赎》。',\n",
       "  '是的，我喜欢听流行音乐。',\n",
       "  '我最喜欢的季节是夏天。',\n",
       "  '是的，我有一只猫。',\n",
       "  '是的，我有一只猫。'],\n",
       " 'history': [None, None, None, None, None, None, None, None, None, None]}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:44:18.181076Z",
     "start_time": "2025-07-17T00:44:13.309447Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from modelscope import AutoModelForCausalLM\n",
    "from modelscope import AutoTokenizer\n",
    "\n",
    "model_name = \"openai-community/gpt2\"\n",
    "gpt_model = AutoModelForCausalLM.from_pretrained(model_name)\n",
    "gpt_tokenizer = AutoTokenizer.from_pretrained(\"openai-community/gpt2\")\n",
    "gpt_tokenizer.pad_token = gpt_tokenizer.eos_token"
   ],
   "id": "6b6a86037ed62719",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: /Users/dadudu/.cache/modelscope/hub/models/openai-community/gpt2\n",
      "Downloading Model from https://www.modelscope.cn to directory: /Users/dadudu/.cache/modelscope/hub/models/openai-community/gpt2\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:44:19.729272Z",
     "start_time": "2025-07-17T00:44:19.717260Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# 大都督周瑜（我的微信: dadudu6789）\n",
    "\n",
    "prompt = \"问题:{}\\n答案:\"\n",
    "# prompt = \"{}\"\n",
    "\n",
    "class ChatBotDataset(Dataset):\n",
    "    def __init__(self, data, max_length=128):\n",
    "        self.data = data\n",
    "        self.encodings = []\n",
    "        for qa in data:  # 遍历文件的每一行\n",
    "            # 将问题+答案拼接为单序列\n",
    "            text = prompt.format(qa['question']) + qa['answer'] + gpt_tokenizer.eos_token\n",
    "            # text = qa['question'] + qa['answer'] + gpt_tokenizer.eos_token\n",
    "            print(text)\n",
    "            encoded = gpt_tokenizer(\n",
    "                text,\n",
    "                max_length=max_length,\n",
    "                padding='max_length',\n",
    "                truncation=True,\n",
    "                return_tensors='pt'\n",
    "            )\n",
    "            input_ids = encoded['input_ids'].squeeze()\n",
    "            self.encodings.append(input_ids)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.encodings)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return self.encodings[idx]\n",
    "\n",
    "chat_dataset = ChatBotDataset(data)\n",
    "data_loader = DataLoader(chat_dataset, batch_size=8, shuffle=True)\n",
    "for batch in data_loader:\n",
    "    print(batch)\n",
    "    break"
   ],
   "id": "26865359b045d785",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "问题:你好，最近怎么样？\n",
      "答案:你好！我最近还不错，谢谢。<|endoftext|>\n",
      "问题:今天天气如何？\n",
      "答案:今天的天气很晴朗。<|endoftext|>\n",
      "问题:你喜欢旅行吗？\n",
      "答案:是的，我非常喜欢旅行。<|endoftext|>\n",
      "问题:你最喜欢的食物是什么？\n",
      "答案:我最喜欢的食物是寿司。<|endoftext|>\n",
      "问题:你有什么兴趣爱好？\n",
      "答案:我喜欢阅读和运动。<|endoftext|>\n",
      "问题:你最喜欢的电影是什么？\n",
      "答案:我最喜欢的电影是《肖申克的救赎》。<|endoftext|>\n",
      "问题:你喜欢听音乐吗？\n",
      "答案:是的，我喜欢听流行音乐。<|endoftext|>\n",
      "问题:你最喜欢的季节是哪个？\n",
      "答案:我最喜欢的季节是夏天。<|endoftext|>\n",
      "问题:你有什么宠物吗？\n",
      "答案:是的，我有一只猫。<|endoftext|>\n",
      "问题:你有宠物吗？\n",
      "答案:是的，我有一只猫。<|endoftext|>\n",
      "tensor([[29785,   106,   165,  ..., 50256, 50256, 50256],\n",
      "        [29785,   106,   165,  ..., 50256, 50256, 50256],\n",
      "        [29785,   106,   165,  ..., 50256, 50256, 50256],\n",
      "        ...,\n",
      "        [29785,   106,   165,  ..., 50256, 50256, 50256],\n",
      "        [29785,   106,   165,  ..., 50256, 50256, 50256],\n",
      "        [29785,   106,   165,  ..., 50256, 50256, 50256]])\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:46:23.990450Z",
     "start_time": "2025-07-17T00:45:06.332821Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "optimizer = torch.optim.Adam(gpt_model.parameters(), lr=0.001)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "gpt_model.to(device)\n",
    "\n",
    "EPOCHS = 50\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "\n",
    "    for input_ids in data_loader:\n",
    "        input_ids = input_ids.to(device)\n",
    "\n",
    "        outputs = gpt_model(\n",
    "            input_ids, labels=input_ids\n",
    "        )\n",
    "\n",
    "        loss = outputs.loss\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if epoch % 10 == 0:\n",
    "            print(f'Epoch {epoch + 1}, Loss: {loss:.4f}')\n",
    "\n"
   ],
   "id": "66f44477426b2348",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`loss_type=None` was set in the config but it is unrecognised.Using the default loss: `ForCausalLMLoss`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 9.6288\n",
      "Epoch 1, Loss: 3.3672\n",
      "Epoch 11, Loss: 0.5160\n",
      "Epoch 11, Loss: 0.6401\n",
      "Epoch 21, Loss: 0.1413\n",
      "Epoch 21, Loss: 0.1014\n",
      "Epoch 31, Loss: 0.0427\n",
      "Epoch 31, Loss: 0.0476\n",
      "Epoch 41, Loss: 0.0275\n",
      "Epoch 41, Loss: 0.0267\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:39:37.643781Z",
     "start_time": "2025-07-17T00:39:37.641282Z"
    }
   },
   "cell_type": "code",
   "source": "print(f'参数量：{sum(p.numel() for p in gpt_model.parameters())}')",
   "id": "8921126c594aa12e",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "参数量：124439808\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:48:33.538732Z",
     "start_time": "2025-07-17T00:48:33.017253Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def generate(sentence, max_length=500):\n",
    "    input_ids = gpt_tokenizer.encode(sentence, return_tensors='pt')\n",
    "    input_ids = input_ids.to(device)\n",
    "    output = gpt_model.generate(\n",
    "        input_ids,\n",
    "        max_length=max_length,\n",
    "        pad_token_id = gpt_tokenizer.eos_token_id,\n",
    "        # do_sample=True,\n",
    "    )\n",
    "    return gpt_tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "\n",
    "input = prompt.format(\"你喜欢听音乐吗\")\n",
    "# input = \"今天天气如何\"\n",
    "print(generate(input)[len(input):])"
   ],
   "id": "df6f0e1df51b6b04",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "是的，我非常喜欢旅行。\n"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 模型本地保存和加载",
   "id": "fbd8aab595720b85"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:56:59.713894Z",
     "start_time": "2025-07-17T00:56:58.767313Z"
    }
   },
   "cell_type": "code",
   "source": "gpt_model.save_pretrained(\"./zhouyu_gpt_model\")",
   "id": "45c221e12b2f0484",
   "outputs": [],
   "execution_count": 31
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:58:25.401064Z",
     "start_time": "2025-07-17T00:58:25.331455Z"
    }
   },
   "cell_type": "code",
   "source": "zhouyu_gpt_model = AutoModelForCausalLM.from_pretrained(\"./zhouyu_gpt_model\")",
   "id": "af65000f645c9f46",
   "outputs": [],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-17T00:58:51.322758Z",
     "start_time": "2025-07-17T00:58:49.594610Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def generate(sentence, model, max_length=500):\n",
    "    input_ids = gpt_tokenizer.encode(sentence, return_tensors='pt')\n",
    "    input_ids = input_ids.to(device)\n",
    "    output = model.generate(\n",
    "        input_ids,\n",
    "        max_length=max_length,\n",
    "        pad_token_id = gpt_tokenizer.eos_token_id,\n",
    "        # do_sample=True,\n",
    "    )\n",
    "    return gpt_tokenizer.decode(output[0], skip_special_tokens=True)\n",
    "\n",
    "input = prompt.format(\"你喜欢听音乐吗\")\n",
    "# input = \"今天天气如何\"\n",
    "print(generate(input, model=zhouyu_gpt_model)[len(input):])"
   ],
   "id": "cb3c555035f43bb6",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "是的，我非常喜欢旅行。\n"
     ]
    }
   ],
   "execution_count": 33
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
